//https://www.freeproxy.world

import { getLogger } from "../utils/log4j";
import cheerioModule from "cheerio";
import { ProxyModel } from "../model";
import axios, { AxiosResponse } from "axios";
import { getCancelTokenSource } from "../utils/proxy-util";
import { MongoDBClient } from "../db";
import { ExtractClientInterface, ProxyInterface, TransformerClientInterface } from "../etl";
import qs from 'qs'

const logger = getLogger(`free-proxy-world`)
export class FreeWorld implements ExtractClientInterface {
	async * extract(): AsyncGenerator<ProxyInterface, void, any> {
		let index = 0
		while (true) {
			const source = getCancelTokenSource()

			setTimeout(() => {
				source.cancel(`请求超时`)
			}, 10000)
			let res: AxiosResponse<never>
			const url = `https://www.freeproxy.world`
			const params = qs.parse(`type=&anonymity=&country=&speed=&port=&page=${++index}`)
			try {
				res = await axios.get(url, { cancelToken: source.token, params: params })
			} catch (error) {
				logger.error(`request ${url} fail`)
				continue
			}
			const $ = cheerioModule.load(res.data)
			const tbody = $('.layui-table tbody')
			const trList = tbody.children().toArray()
			if (trList.length === 0) break

			for (let i = 1; i < trList.length; i += 2) {
				const tr = trList[i]
				const tdList = $('td', tr).toArray()
				const ip = $(tdList[0]).text().replace(/[\n]/g, '')
				const port = $(tdList[1]).text().replace(/[\n]/g, '')
				const country = $(tdList[2]).text().replace(/[\n]/g, '')
				const city = $(tdList[3]).text().replace(/[\n]/g, '')
				const type = $(tdList[5]).text().replace(/[\n]/g, '')
				const proxy: ProxyInterface = {
					ip: ip,
					port: Number(port),
					country: country,
					city: city,
					protocol: type
				}
				yield proxy
			}
		}
	}
}